#!/bin/bash

#SBATCH -J test-paradis
#SBATCH -N 1
#SBATCH --ntasks-per-node 32
#SBATCH -p normal
#SBATCH -o paradis-%j.log
#SBATCH -e paradis-%j.err
#SBATCH --comment=WRF

module purge
source ~/.bashrc
#unset MPIHOME
#unset LD_LIBRARY_PATH

ulimit -c unlimited

mpi_env=ompi
if [[ $mpi_env == impi* ]]; then intelMPI=1; else intelMPI=0; fi
sev $mpi_env 
module list

hostfile=hostfile.${SLURM_JOB_ID}
srun hostname -s | sort -n > $hostfile
cat $hostfile
ncpu=`srun hostname -s | wc -l`
echo "ncpu=$ncpu"

exeName=paradis
exeDir=${HOME}/DWei/paradis/bin
if [[ ! -f ${exeDir}/$exeName ]]; then echo "can not find ${exeDir}/$exeName"; exit 1; fi

ctrlFiles=`(find -type f -name "Copper.ctrl" && find -type f -name "rs[0-9][0-9][0-9][0-9]" && find -type f -name "restart.cn")`
if [[ -z ${ctrlFiles} ]] ; then exit 1; fi
ctrlFile=`ls -t ${ctrlFiles} | head -n 1`

if [[ ! -f ${ctrlFile} ]]; then exit 1; fi;

dataFiles=`(find -type f -name "Copper.data" && find -type f -name "rs[0-9][0-9][0-9][0-9].data" && find -type f -name "restart.data")`
if [[ -z ${dataFiles} ]] ; then exit 1; fi
dataFile=`ls -t ${dataFiles} | head -n 1`
nodeCount=`val nodeCount ${dataFile}`
if [[ `echo "${nodeCount}>30000" | bc` -eq 1 ]] ; then ncells=16;
elif [[ `echo "${nodeCount}>3000" | bc` -eq 1 ]] ; then ncells=8;
else ncells=4; fi
changeVal numXcells ${ncells} ${ctrlFile}
changeVal numYcells ${ncells} ${ctrlFile}
changeVal numZcells ${ncells} ${ctrlFile}
changeVal dirname \"res.${SLURM_JOB_NAME}-${SLURM_JOB_ID}\"  ${ctrlFile}

oCtrlFile=`find -type f -name "Copper.ctrl"`
oDataFile=Copper.data
debughash=""

case $ncpu in
    1)
    numXdoms=1;
    numYdoms=1;
    numZdoms=1;;
    2)
    numXdoms=1;
    numYdoms=1;
    numZdoms=2;;
    4)
    numXdoms=2;
    numYdoms=2;
    numZdoms=1;;
    8)
    numXdoms=2;
    numYdoms=2;
    numZdoms=2;;
    16)
    numXdoms=2;
    numYdoms=2;
    numZdoms=4;;
    32)
    numXdoms=4;
    numYdoms=4;
    numZdoms=2;;
    64)
    numXdoms=4;
    numYdoms=4;
    numZdoms=4;;
    128)
    numXdoms=4;
    numYdoms=4;
    numZdoms=8;;
    256)
    numXdoms=4;
    numYdoms=8;
    numZdoms=8;;
esac
sed -i "/numXdoms/cnumXdoms = ${numXdoms}" ${ctrlFile}
sed -i "/numYdoms/cnumYdoms = ${numYdoms}" ${ctrlFile}
sed -i "/numZdoms/cnumZdoms = ${numZdoms}" ${ctrlFile}


export PATH=$exeDir:$PATH
export MKL_NUM_THREADS=1
export OMP_NUM_THREADS=1

#echo "I_MPI_ROO=$I_MPI_ROOT"
#if [[ `find $I_MPI_ROOT -type f -name "mpirun"` ]]; then echo "mpirun is found" >> paradis-${SLURM_JOB_ID}.log; fi
#echo "PATH=${PATH}"  >> paradis-${SLURM_JOB_ID}.log;
#echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}"  >> paradis-${SLURM_JOB_ID}.log;

resCtrlFile=`find -type f -name "restart.cn"`
resDataFile=`find -type f -name "restart.data"`

echo "JobFileName=$0"
MPIRUN_EXE=`which mpirun`

#mpirun -n ${ncpus} -machinefile $hostfile ${exe} ./$ctrlFile
#srun --mpi=pmix_v3 $exe ./Copper.ctrl 
exe=$exeDir/$exeName
if [[ ${intelMPI} == "1" ]]; then
    srun --mpi=pmix_v3  $exeDir/$exeName ./${ctrlFile}
else
    MPIRUN_EXE=$(which mpirun)
    $MPIRUN_EXE -np $ncpu -machinefile $hostfile $exe -n 1 $ctrlFile
fi
rm -f ${hostfile}

if [[ $resCtrlFile == $ctrlFile ]]; then rm $resCtrlFile; fi
if [[ $resDataFile == $dataFile ]]; then rm $resDataFile; fi

resCnFile=`find -type f -name "restart.cn"`;
resDataFile=`find -type f -name "restart.data"`;

relaxed=1;
if [[ ${oCtrlFile} && ${resCnFile} && ${resDataFile} ]] ; then
    if [[ `val loadType ${resCnFile}` -eq 0 ]] ; then
        changeVal loadType 1 ${oCtrlFile};
        changeVal maxstep 4000000 ${oCtrlFile};
        \cp -f ${resDataFile} ${oDataFile}
        rm ${resCnFile};
        relaxed=0;
    fi
fi

resCnFile=`find -type f -name "restart.cn"`;
if [[ ${resCnFile} || `find -type f -name "Finish"` ]] ; then
    echo "Job ${SLURM_JOB_NAME} has been sucessfully completed !" >> paradis-${SLURM_JOB_ID}.log
elif [[ `grep "Fatal" paradis-${SLURM_JOB_ID}.log` ]] ; then
    echo "ERROR: Job ${SLURM_JOB_NAME} has somthing wrong: " >>  paradis-${SLURM_JOB_ID}.log
    echo "${SLURM_JOB_ID} ${SLURM_JOB_NAME} $ncpu $cdir ERROR!! " >> ${HOME}/sf.log  
    inf=`grep "Fatal" paradis-${SLURM_JOB_ID}.log | grep "Pack"`;
  # mv $ctrlFile fatal-bak-${SLURM_JOB_ID}
    qsub ${SLURM_JOB_NAME} $ncpu ~/$0
elif [[ $relaxed == "1" ]]; then
    echo "Warning: Job ${SLURM_JOB_NAME}: segmentation fault" >>  paradis-${SLURM_JOB_ID}.log
    cdir=`pwd`
    time=`date`
    echo "${SLURM_JOB_ID} ${SLURM_JOB_NAME} $ncpu $cdir $time" >> ${HOME}/sf.log  
    qsub ${SLURM_JOB_NAME} $ncpu ~/$0
else
    echo "Next step: loadType=1" >>  paradis-${SLURM_JOB_ID}.log
    qsub ${SLURM_JOB_NAME} $ncpu ~/$0
fi
